Weekly notes text analysis

library(readr)
library(tibble)
library(tidyr)

Attaching package: ‘tidyr’

The following object is masked from ‘package:Matrix’:

    expand

library(dplyr)

Attaching package: ‘dplyr’

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union

library(tidytext)
library(ggplot2)
library(tm)

Loading required package: NLP

Attaching package: ‘NLP’

The following object is masked from ‘package:ggplot2’:

    annotate

library(topicmodels)
library(stringr)

weeknotes_ryan <- as_tibble(read_csv('weeknotes_ryan.csv', col_names = FALSE))

Parsed with column specification:
cols(
  X1 = col_character(),
  X2 = col_character(),
  X3 = col_character(),
  X4 = col_character(),
  X5 = col_character(),
  X6 = col_character(),
  X7 = col_character(),
  X8 = col_character(),
  X9 = col_character(),
  X10 = col_character()
)

names(weeknotes_ryan) <- c( "s02e01",
                            "s01e09",
                            "s01e08",
                            "s01e07",
                            "s01e06",
                            "s01e05",
                            "s01e04",
                            "s01e03",
                            "s01e02",      
                            "s01e01"
       )

weeknotes_ryan %>% 
  gather("episode", "text", 1:10) -> tidy_notes

tidy_notes

tidy_notes %>% 
  mutate(text2 = gsub("[.]", " ", text)) -> tidier_notes

tidier_notes %>% 
  unnest_tokens(word, text2) -> tidy_tokens

tidy_tokens

data("stop_words")
other_stop_words <- c("i’d", "i’m", "i’ve")

tidy_tokens %>% 
  anti_join(stop_words) %>% 
  filter(!(word %in% other_stop_words)) -> tidier_wo_stopwords

Joining, by = "word"

tidier_wo_stopwords %>% 
  filter(str_detect(word, regex("i’d", ignore_case = TRUE)))

tidier_wo_stopwords %>% 
  filter(episode == 's01e01') %>% 
  count(word, sort=TRUE)

tidier_wo_stopwords %>% 
  filter(episode == 's02e01') %>% 
  count(word, sort=TRUE)

library(ggplot2)
tidier_wo_stopwords %>%
  count(word, sort = TRUE) %>%
  filter(n > 30) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(word, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

tidier_wo_stopwords

tidier_wo_stopwords %>%
  count(episode, word, sort = TRUE) %>%
  ungroup() -> kk
kk %>% 
  bind_tf_idf(word, episode, n) -> tidier_tf_idf 

tidier_tf_idf %>% 
  arrange(desc(tf_idf))

tidier_tf_idf %>%
  arrange(desc(tf_idf)) %>%
  mutate(word = factor(word, levels = rev(unique(word)))) -> plot_episodes_tf_idf

plot_episodes_tf_idf %>% 
  group_by(episode) %>% 
  top_n(7) %>% 
  ungroup %>%
  ggplot(aes(word, tf_idf, fill = episode)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~episode, ncol = 2, scales = "free") +
  coord_flip()

Selecting by tf_idf

tidier_notes %>% 
  unnest_tokens(bigram, text, token = "ngrams", n = 2) -> bigrams

bigrams %>%
  count(bigram, sort = TRUE)

bigrams_separated <- bigrams %>%
  separate(bigram, c("word1", "word2"), sep = " ")
bigrams_filtered <- bigrams_separated %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word) %>% 
  filter(!word2 %in% other_stop_words) %>% 
  filter(!word2 %in% other_stop_words)
  
# new bigram counts:
bigram_counts <- bigrams_filtered %>% 
  count(word1, word2, sort = TRUE)
bigram_counts

tidier_notes %>% 
  unnest_tokens(trigram, text, token = "ngrams", n = 3) -> trigrams
trigrams_separated <- trigrams %>%
  separate(trigram, c("word1", "word2", "word3"), sep = " ")
trigrams_filtered <- trigrams_separated %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word3 %in% stop_words$word) %>% 
  filter(!word1 %in% other_stop_words) %>% 
  filter(!word3 %in% other_stop_words)
  
# new bigram counts:
trigram_counts <- trigrams_filtered %>% 
  count(word1, word2, word3, sort = TRUE)
trigram_counts

tidier_tf_idf %>% 
  cast_dtm(document = episode,term = word, value = n) -> dtm_matrix

dtm_matrix

<<DocumentTermMatrix (documents: 10, terms: 2204)>>
Non-/sparse entries: 4623/17417
Sparsity           : 79%
Maximal term length: 19
Weighting          : term frequency (tf)

tidier_tf_idf %>% 
  cast_sparse(episode, word, n) -> sparse_matrix

str(sparse_matrix)

Formal class 'dgCMatrix' [package "Matrix"] with 6 slots
  ..@ i       : int [1:4623] 0 1 2 3 4 5 6 7 8 9 ...
  ..@ p       : int [1:2205] 0 10 20 24 34 42 52 62 72 82 ...
  ..@ Dim     : int [1:2] 10 2204
  ..@ Dimnames:List of 2
  .. ..$ : chr [1:10] "s01e05" "s01e09" "s01e08" "s01e01" ...
  .. ..$ : chr [1:2204] "data" "call" "event" "meeting" ...
  ..@ x       : num [1:4623] 31 26 25 23 20 20 17 17 14 11 ...
  ..@ factors : list()

txt_lda <- LDA(dtm_matrix, k = 2, control = list(seed = 1234))

txt_topics <- tidy(txt_lda, matrix = "beta")
top_terms <- txt_topics %>%
  group_by(topic) %>%
  top_n(10, beta) %>%
  ungroup() %>%
  arrange(topic, -beta)
top_terms %>%
  mutate(term = reorder(term, beta)) %>%
  ggplot(aes(term, beta, fill = factor(topic))) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~ topic, scales = "free") +
  coord_flip()